The goals / steps of this project are the following:
import numpy as np
import cv2
import glob
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
%matplotlib inline
#matplotlib qt
def show_images(images):
num_img = len(images)
#print(num_img)
#plt.figure(figsize=(15,5))
#fig, axes = plt.subplots(num_img, 1)
#for axe, i in zip(axes.flat, range(num_img)):
# print(i)
# axe.imshow(images[i])
# #plt.imshow(images[i])
for i in range(num_img):
#print("i:",i)
plt.figure(i)
#plt.subplot(num_img, 1, i+1)
plt.imshow(images[i])
#plt.imshow(images[i])
#plt.show()
def show_two_images(img1, img2, title1="Original Image", title2="Result Image"):
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(img1)
ax1.set_title(title1, fontsize=50)
ax2.imshow(img2)
ax2.set_title(title2, fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)
def show_pair_images(images1, images2):
for i in range(len(images1)):
show_two_images(images1[i], imges2[i])
First, I'll compute the camera calibration using chessboard images
# a function that takes an image, object points, and image points
# performs the camera calibration, image distortion correction and
# returns the undistorted image
def cal_undistort(img, objpoints, imgpoints):
gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, gray.shape[::-1], None, None)
img_dst = cv2.undistort(img, mtx, dist, None, mtx)
return img_dst
def compute_camera_calibration(images,chessboard_size_x=9, chessboard_size_y=6):
# prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
objp = np.zeros((chessboard_size_y*chessboard_size_x,3), np.float32)
objp[:,:2] = np.mgrid[0:chessboard_size_x,0:chessboard_size_y].T.reshape(-1,2)
# Arrays to store object points and image points from all the images.
objpoints = [] # 3d points in real world space
imgpoints = [] # 2d points in image plane.
# Make a list of calibration images
images_converted = []
# Step through the list and search for chessboard corners
for fname in images:
#print(fname)
img = cv2.imread(fname)
img_converted = np.copy(img)
gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
# Find the chessboard corners
ret, corners = cv2.findChessboardCorners(gray, (chessboard_size_x,chessboard_size_y),None)
# If found, add object points, image points
if ret == True:
objpoints.append(objp)
imgpoints.append(corners)
# Draw and display the corners
img_converted = cv2.drawChessboardCorners(img_converted, (chessboard_size_x,chessboard_size_y), corners, ret)
images_converted.append(img_converted)
# Undistort the original image using objpoints and imgpoints
img_undistorted = cal_undistort(img, objpoints, imgpoints)
show_two_images(img_converted, img_undistorted)
return objpoints, imgpoints
camera_images = glob.glob('./camera_cal/calibration*.jpg')
objpoints, imgpoints = compute_camera_calibration(camera_images)
use chessboard images to obtain image points and object points, and then compute the calibration and undistortion.
test_img = cv2.imread('./camera_cal/calibration2.jpg')
undistorted = cal_undistort(test_img, objpoints, imgpoints)
show_two_images(test_img, undistorted, title2="Undistorted Image")
import numpy as np
import cv2
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
# Edit this function to create your own pipeline.
def pipeline_thresh(img, s_thresh=(170, 255), sx_thresh=(20, 100)):
img = np.copy(img)
# Convert to HLS color space and separate the V channel
hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS).astype(np.float)
l_channel = hls[:,:,1]
s_channel = hls[:,:,2]
# Grayscale image
# NOTE: we already saw that standard grayscaling lost color information for the lane lines
# Explore gradients in other colors spaces / color channels to see what might work better
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
# Sobel x
#sobelx = cv2.Sobel(l_channel, cv2.CV_64F, 1, 0) # Take the derivative in x
sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0) # Take the derivative in x
abs_sobelx = np.absolute(sobelx) # Absolute x derivative to accentuate lines away from horizontal
scaled_sobel = np.uint8(255*abs_sobelx/np.max(abs_sobelx))
# Threshold x gradient
sxbinary = np.zeros_like(scaled_sobel)
sxbinary[(scaled_sobel >= sx_thresh[0]) & (scaled_sobel <= sx_thresh[1])] = 1
# Threshold color channel
s_binary = np.zeros_like(s_channel)
s_binary[(s_channel >= s_thresh[0]) & (s_channel <= s_thresh[1])] = 1
# Stack each channel
# Note color_binary[:, :, 0] is all 0s, effectively an all black image. It might
# be beneficial to replace this channel with something else.
color_binary = np.dstack(( np.zeros_like(sxbinary), sxbinary, s_binary)) * 255
# Combine the two binary thresholds
combined_binary = np.zeros_like(sxbinary)
combined_binary[(s_binary == 1) | (sxbinary == 1)] = 1
#combined_binary[(s_binary == 1)] = 1
return combined_binary
#return color_binary
#return gray
#image = mpimg.imread('./test_images/test5.jpg')
test_image = mpimg.imread('./test_images/straight_lines1.jpg')
image_threshed = pipeline_thresh(test_image)
show_two_images(test_image, image_threshed, title2="Thresh Pipeline Result")
test_img = mpimg.imread('./issue_images/org_video_image_20171217_10_22_06.jpg')
test_output = pipeline_thresh(test_img, s_thresh=(170, 255), sx_thresh=(5, 100))
show_two_images(test_img, test_output, title2="Magnitude Image")
undistorted = cal_undistort(test_image, objpoints, imgpoints)
show_two_images(test_image, undistorted, title2="Undistorted Image")
Y_TOP = 435#440
Y_BTM = 668
X_CTR = 640#640 = 1280/2
X_TOP_WDT = 48#45#50#55#65#50#50#60#60
X_BTM_WDT = 1000#1280#1000#1000#900#850
# less space is used
Y_TOP = 450#435#440
X_TOP_WDT = 120#48#45#50#55#65#50#50#60#60
TOP_LEFT = (X_CTR - X_TOP_WDT/2 , Y_TOP)
TOP_RIGHT = (X_CTR + X_TOP_WDT/2, Y_TOP)
BTM_LEFT = (X_CTR - X_BTM_WDT/2 , Y_BTM)
BTM_RIGHT = (X_CTR + X_BTM_WDT/2, Y_BTM)
SRC_TRAPEZOID_ORG_ORDR = [TOP_LEFT, TOP_RIGHT, BTM_RIGHT, BTM_LEFT]
#SRC_TRAPEZOID_WRP = [TOP_LEFT, TOP_RIGHT, BTM_LEFT, BTM_RIGHT]
def pipeline_warp(image, offsetX = 50, offsetY = 0):
offset = 100 # offset for dst points
# Grab the image shape
img_size = (image.shape[1], image.shape[0])
w = img_size[1]
h = img_size[1]
offsetX = 30 # test
src = np.float32(SRC_TRAPEZOID_ORG_ORDR)#SRC_RECTANGLE)
#error: ..\..\..\modules\imgproc\src\imgwarp.cpp:6101: error: (-215) _src.total() > 0 in function cv::warpPerspective
#src = np.float32([corners[0],corners[7],corners[40],corners[47]])#([[,],[,],[,],[,]])
# c) define 4 destination points dst = np.float32([[,],[,],[,],[,]])
# For destination points, I'm arbitrarily choosing some points to be
# a nice fit for displaying our warped result
# again, not exact, but close enough for our purposes
#dst = np.float32([[offset, offset], [img_size[0]-offset, offset],
# [img_size[0]-offset, img_size[1]-offset],
# [offset, img_size[1]-offset]])
#dst = np.float32([[offsetX, offsetY], [w - offsetX, offsetY],
# [offsetX, h - offsetY], [w - offsetX, h - offsetY]])
dst = np.float32([[offsetX, offsetY], [w - offsetX, offsetY],
[w - offsetX, h - offsetY],
[offsetX, h - offsetY]])
#dst = np.float32([[,],[,],[,],[,]])
# d) use cv2.getPerspectiveTransform() to get M, the transform matrix
M = cv2.getPerspectiveTransform(src, dst)
# e) use cv2.warpPerspective() to warp your image to a top-down view
#warped = cv2.warpPerspective(img, M, img_size, flags=cv2.INTER_LINEAR)
warped = cv2.warpPerspective(image,M,(h, w))
Minv = cv2.getPerspectiveTransform(dst, src)
return warped, M, Minv
#test_img = mpimg.imread('./test_images/straight_lines1.jpg')
test_img = mpimg.imread('./test_images/test1.jpg')
image_threshed = pipeline_thresh(test_img)
image_warped, M, Minv = pipeline_warp(image_threshed)
show_two_images(test_img, image_threshed, title2="Binary Image")
show_two_images(test_img, image_warped, title2="Warped Image")
TODO; Line class is not used.
# Define a class to receive the characteristics of each line detection
class Line():
def __init__(self):
# was the line detected in the last iteration?
self.detected = False
# x values of the last n fits of the line
self.recent_xfitted = []
#average x values of the fitted line over the last n iterations
self.bestx = None
#polynomial coefficients averaged over the last n iterations
self.best_fit = None
#polynomial coefficients for the most recent fit
self.current_fit = [np.array([False])]
self.prev_fit = [np.array([False])]
#radius of curvature of the line in some units
self.radius_of_curvature = None
#distance in meters of vehicle center from the line
self.line_base_pos = None
#difference in fit coefficients between last and new fits
self.diffs = np.array([0,0,0], dtype='float')
#x values for detected line pixels
self.allx = None
#y values for detected line pixels
self.ally = None
def calc_diffs():
self.diffs[0] = self.current_fit[0] - self.prev_fit[0]
self.diffs[1] = self.current_fit[1] - self.prev_fit[1]
self.diffs[2] = self.current_fit[2] - self.prev_fit[2]
def update_average_and_best(self):
#average x values of the fitted line over the last n iterations
self.bestx = None
#polynomial coefficients averaged over the last n iterations
self.best_fit = None
def compare_last_and_recent(self):
#difference in fit coefficients between last and new fits
self.diffs = np.array([0,0,0], dtype='float')
def add_recent_data(self, current_fit, detected_x, detected_y):
# was the line detected in the last iteration?
#self.detected = detected
#polynomial coefficients for the most recent fit
self.current_fit = current_fit#[np.array([False])]
# x values of the last n fits of the line
self.recent_xfitted.append(current_fit)# = [] #?
#x values for detected line pixels
self.allx = detected_x #? None
#y values for detected line pixels
self.ally = detected_y #? None
# Assuming you have created a warped binary image called "binary_warped"
def find_lines(binary_warped):
# Choose the number of sliding windows
nwindows = 9
# Set the width of the windows +/- margin
margin = 100
#margin = 150
# Set minimum number of pixels found to recenter window
minpix = 50
# Take a histogram of the bottom half of the image
#histogram = np.sum(binary_warped[binary_warped.shape[0]/2:,:], axis=0)
histogram = np.sum(binary_warped[np.int(binary_warped.shape[0]/2):,:], axis=0)
# Create an output image to draw on and visualize the result
out_img = np.dstack((binary_warped, binary_warped, binary_warped))*255
# Find the peak of the left and right halves of the histogram
# These will be the starting point for the left and right lines
midpoint = np.int(histogram.shape[0]/2)
leftx_base = np.argmax(histogram[:midpoint])
rightx_base = np.argmax(histogram[midpoint:]) + midpoint
# Set height of windows
window_height = np.int(binary_warped.shape[0]/nwindows)
# Identify the x and y positions of all nonzero pixels in the image
nonzero = binary_warped.nonzero()
nonzeroy = np.array(nonzero[0])
nonzerox = np.array(nonzero[1])
# Current positions to be updated for each window
leftx_current = leftx_base
rightx_current = rightx_base
# Create empty lists to receive left and right lane pixel indices
left_lane_inds = []
right_lane_inds = []
# Step through the windows one by one
right_stop = False
left_stop = False
for window in range(nwindows):
# Identify window boundaries in x and y (and right and left)
win_y_low = binary_warped.shape[0] - (window+1)*window_height
win_y_high = binary_warped.shape[0] - window*window_height
win_xleft_low = leftx_current - margin
win_xleft_high = leftx_current + margin
win_xright_low = rightx_current - margin
win_xright_high = rightx_current + margin
# Draw the windows on the visualization image
cv2.rectangle(out_img,(win_xleft_low,win_y_low),(win_xleft_high,win_y_high),
(0,255,0), 2)
cv2.rectangle(out_img,(win_xright_low,win_y_low),(win_xright_high,win_y_high),
#(0,255,0), 2)
(0,0,255), 2)
# Identify the nonzero pixels in x and y within the window
good_left_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) &
(nonzerox >= win_xleft_low) & (nonzerox < win_xleft_high)).nonzero()[0]
good_right_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) &
(nonzerox >= win_xright_low) & (nonzerox < win_xright_high)).nonzero()[0]
# Append these indices to the lists
##if left_stop == False:
left_lane_inds.append(good_left_inds)
##if right_stop == False:
right_lane_inds.append(good_right_inds)
# If you found > minpix pixels, recenter next window on their mean position
if len(good_left_inds) > minpix:
leftx_current = np.int(np.mean(nonzerox[good_left_inds]))
if len(good_right_inds) > minpix:
rightx_current = np.int(np.mean(nonzerox[good_right_inds]))
# If
##if (len(nonzerox[good_left_inds]) < 1 and window > nwindows//2) :
## left_stop = True
##if (len(nonzerox[good_right_inds]) < 1 and window > nwindows//2) :
## right_stop = True
#print(len(nonzerox[good_left_inds]), nwindows//2, window,( window > nwindows//2) )
# Concatenate the arrays of indices
left_lane_inds = np.concatenate(left_lane_inds)
right_lane_inds = np.concatenate(right_lane_inds)
# Extract left and right line pixel positions
leftx = nonzerox[left_lane_inds]
lefty = nonzeroy[left_lane_inds]
rightx = nonzerox[right_lane_inds]
righty = nonzeroy[right_lane_inds]
# Fit a second order polynomial to each
left_fit = np.polyfit(lefty, leftx, 2)
right_fit = np.polyfit(righty, rightx, 2)
right_line = Line()
left_line = Line()
right_line.update_average_and_best()
right_line.add_recent_data(right_fit, rightx, righty)
left_line.add_recent_data(left_fit, leftx, lefty)
# was the line detected in the last iteration?
right_line.detected = False
# x values of the last n fits of the line
right_line.recent_xfitted = []
#average x values of the fitted line over the last n iterations
right_line.bestx = None
#polynomial coefficients averaged over the last n iterations
right_line.best_fit = None
#polynomial coefficients for the most recent fit
right_line.current_fit = right_fit #[np.array([False])]
#radius of curvature of the line in some units
right_line.radius_of_curvature = None
#distance in meters of vehicle center from the line
right_line.line_base_pos = None
#difference in fit coefficients between last and new fits
right_line.diffs = np.array([0,0,0], dtype='float')
return left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img
Now I know where the lines are you have a fit! In the next frame of video I don't need to do a blind search again, but instead I can just search in a margin around the previous line position.
# Assume you now have a new warped binary image
# from the next frame of video (also called "binary_warped")
# It's now much easier to find line pixels!
def find_lines_again(binary_warped, left_fit, right_fit):
nonzero = binary_warped.nonzero()
nonzeroy = np.array(nonzero[0])
nonzerox = np.array(nonzero[1])
margin = 100
margin = 50
#margin = 30
left_lane_inds = ((nonzerox > (left_fit[0]*(nonzeroy**2) + left_fit[1]*nonzeroy +
left_fit[2] - margin)) & (nonzerox < (left_fit[0]*(nonzeroy**2) +
left_fit[1]*nonzeroy + left_fit[2] + margin)))
right_lane_inds = ((nonzerox > (right_fit[0]*(nonzeroy**2) + right_fit[1]*nonzeroy +
right_fit[2] - margin)) & (nonzerox < (right_fit[0]*(nonzeroy**2) +
right_fit[1]*nonzeroy + right_fit[2] + margin)))
# Again, extract left and right line pixel positions
leftx = nonzerox[left_lane_inds]
lefty = nonzeroy[left_lane_inds]
rightx = nonzerox[right_lane_inds]
righty = nonzeroy[right_lane_inds]
# Fit a second order polynomial to each
left_fit = np.polyfit(lefty, leftx, 2)
right_fit = np.polyfit(righty, rightx, 2)
# Generate x and y values for plotting
ploty = np.linspace(0, binary_warped.shape[0]-1, binary_warped.shape[0] )
left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
return left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img
# Calculate the new radii of curvature
def calc_radii_of_curvature(fit_cr, y_eval, ym_per_pix):
curverad = ((1 + (2*fit_cr[0]*y_eval*ym_per_pix + fit_cr[1])**2)**1.5) / np.absolute(2*fit_cr[0])
return curverad
#left_curverad = ((1 + (2*left_fit_cr[0]*y_eval*ym_per_pix + left_fit_cr[1])**2)**1.5) / np.absolute(2*left_fit_cr[0])
#right_curverad = ((1 + (2*right_fit_cr[0]*y_eval*ym_per_pix + right_fit_cr[1])**2)**1.5) / np.absolute(2*right_fit_cr[0])
# Define y-value where we want radius of curvature
# I'll choose the maximum y-value, corresponding to the bottom of the image
def calc_curverad(ploty, fit):
y_eval = np.max(ploty)
curverad = ((1 + (2*fit[0]*y_eval + fit[1])**2)**1.5) / np.absolute(2*fit[0])
return curverad
# Example values: 1926.74 1908.48
def calc_radius_in_meter(y_eval, left_fit_cr, right_fit_cr):
# Define conversions in x and y from pixels space to meters
ym_per_pix = 30/720 # meters per pixel in y dimension
xm_per_pix = 3.7/700 # meters per pixel in x dimension
# Calculate the new radii of curvature
left_curverad = ((1 + (2*left_fit_cr[0]*y_eval*ym_per_pix + left_fit_cr[1])**2)**1.5) / np.absolute(2*left_fit_cr[0])
right_curverad = ((1 + (2*right_fit_cr[0]*y_eval*ym_per_pix + right_fit_cr[1])**2)**1.5) / np.absolute(2*right_fit_cr[0])
# Now our radius of curvature is in meters
return left_curverad, right_curverad
# Example values: 632.1 m 626.2 m
def calc_vehicle_position(left_fit, right_fit, image):
#y_btm = 720#
y_btm = image.shape[0] # Y
#print("y_btm", y_btm)
#image_mid = 720//2
image_mid = image.shape[1]/2 # X
#print("image_mid", image_mid)
x_left = left_fit[0]*y_btm**2 + left_fit[1]*y_btm + left_fit[2]
x_right = right_fit[0]*y_btm**2 + right_fit[1]*y_btm + right_fit[2]
#print("x_left", x_left)
#print("x_right", x_right)
lane_width = x_right - x_left
x_mid = x_left + (lane_width)//2
#print("x_mid", x_mid)
vehicle_position = image_mid - x_mid
LANE_WIDTH_METER = 3.4
meter_per_pixel = LANE_WIDTH_METER/lane_width
vehicle_position_meter = vehicle_position*meter_per_pixel
return vehicle_position_meter#vehicle_position
def put_text(image, text, margin_from_edge = 5, font = cv2.FONT_HERSHEY_PLAIN, font_size = 0.6, color = (255,255,0)):
#w = image.shape[0]
#h = image.shape[1]
start_x = margin_from_edge
start_y = margin_from_edge
cv2.putText(image,text,(start_x,start_y),font, font_size,color)
def draw_rectangle(img):
#result = cv.rectangle(img, (10, 50), (50, 150), (255, 0, 0), 3, 4)
#image = cv2.polylines(img, np.float32(SRC_RECTANGLE), True, (0, 0, 255), 5)
pts = np.array(SRC_TRAPEZOID_ORG_ORDR, np.int32)
pts = pts.reshape((-1, 1, 2))
#pts = np.float32(SRC_RECTANGLE)
copy = img.copy()
cv2.polylines(copy, [pts],True,(255,0,0), thickness=2)
return copy
#X_CENTER = 410
#Y_CENTER = 610
#X_BOTTOM = 660
#X_OFFSET = 5
#Y_OFFSET = 400
class LineProjector():
def __init__(self, Minv):
self.Minv = Minv
def project_lines(self, original_image, warped, left_fit, right_fit):
ploty = np.linspace(0, warped.shape[0]-1, warped.shape[0] )
left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
# Create an image to draw the lines on
warp_zero = np.zeros_like(warped).astype(np.uint8)
color_warp = np.dstack((warp_zero, warp_zero, warp_zero))
# Recast the x and y points into usable format for cv2.fillPoly()
pts_left = np.array([np.transpose(np.vstack([left_fitx, ploty]))])
pts_right = np.array([np.flipud(np.transpose(np.vstack([right_fitx, ploty])))])
pts = np.hstack((pts_left, pts_right))
# Draw the lane onto the warped blank image
cv2.fillPoly(color_warp, np.int_([pts]), (0,255, 0))
# Warp the blank back to original image space using inverse perspective matrix (Minv)
#newwarp = cv2.warpPerspective(color_warp, self.Minv, (image.shape[1], image.shape[0]))
newwarp = cv2.warpPerspective(color_warp, self.Minv, (original_image.shape[1], original_image.shape[0]))
# Combine the result with the original image
result = cv2.addWeighted(original_image, 1, newwarp, 0.3, 0)
return result
# Generate x and y values for plotting
def visualize_lines(warped, left_fit, right_fit, left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img):
ploty = np.linspace(0, warped.shape[0]-1, warped.shape[0] )
out_img[nonzeroy[left_lane_inds], nonzerox[left_lane_inds]] = [255, 0, 0]
out_img[nonzeroy[right_lane_inds], nonzerox[right_lane_inds]] = [0, 0, 255]
plt.imshow(out_img)
# Generate x and y values for plotting
left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
plt.plot(left_fitx, ploty, color='yellow')
plt.plot(right_fitx, ploty, color='yellow')
#print(warped.shape[0], warped.shape[1])
plt.xlim(0, 1280)
plt.ylim(720, 0)
test_img = mpimg.imread('./test_images/test2.jpg')
image_threshed = pipeline_thresh(test_img)
binary_warped, M, Minv = pipeline_warp(image_threshed)
left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img = find_lines(binary_warped)
visualize_lines(binary_warped, left_fit, right_fit, left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img)
#print("left:", calc_radii_of_curvature(left_fit, y_eval, ym_per_pix))
#print("riht:",calc_radii_of_curvature(right_fit, y_eval, ym_per_pix))
test_img = mpimg.imread('./test_images/test2.jpg')
image_threshed = pipeline_thresh(test_img)
binary_warped, M, Minv = pipeline_warp(image_threshed)
left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img = find_lines_again(binary_warped, left_fit, right_fit)
visualize_lines(binary_warped, left_fit, right_fit, left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img)
TODO: Convolution is not used in this solution.
def window_mask(width, height, img_ref, center,level):
output = np.zeros_like(img_ref)
output[int(img_ref.shape[0]-(level+1)*height):int(img_ref.shape[0]-level*height),max(0,int(center-width/2)):min(int(center+width/2),img_ref.shape[1])] = 1
return output
def find_window_centroids(image, window_width, window_height, margin):
window_centroids = [] # Store the (left,right) window centroid positions per level
window = np.ones(window_width) # Create our window template that we will use for convolutions
# First find the two starting positions for the left and right lane by using np.sum to get the vertical image slice
# and then np.convolve the vertical image slice with the window template
# Sum quarter bottom of image to get slice, could use a different ratio
l_sum = np.sum(image[int(3*image.shape[0]/4):,:int(image.shape[1]/2)], axis=0)
l_center = np.argmax(np.convolve(window,l_sum))-window_width/2
r_sum = np.sum(image[int(3*image.shape[0]/4):,int(image.shape[1]/2):], axis=0)
r_center = np.argmax(np.convolve(window,r_sum))-window_width/2+int(image.shape[1]/2)
# Add what we found for the first layer
window_centroids.append((l_center,r_center))
# Go through each layer looking for max pixel locations
for level in range(1,(int)(image.shape[0]/window_height)):
# convolve the window into the vertical slice of the image
image_layer = np.sum(image[int(image.shape[0]-(level+1)*window_height):int(image.shape[0]-level*window_height),:], axis=0)
conv_signal = np.convolve(window, image_layer)
# Find the best left centroid by using past left center as a reference
# Use window_width/2 as offset because convolution signal reference is at right side of window, not center of window
offset = window_width/2
l_min_index = int(max(l_center+offset-margin,0))
l_max_index = int(min(l_center+offset+margin,image.shape[1]))
l_center = np.argmax(conv_signal[l_min_index:l_max_index])+l_min_index-offset
# Find the best right centroid by using past right center as a reference
r_min_index = int(max(r_center+offset-margin,0))
r_max_index = int(min(r_center+offset+margin,image.shape[1]))
r_center = np.argmax(conv_signal[r_min_index:r_max_index])+r_min_index-offset
# Add what we found for that layer
window_centroids.append((l_center,r_center))
return window_centroids
def gen_output(window_centroids, warped):
# If we found any window centers
output = None
if len(window_centroids) > 0:
# Points used to draw all the left and right windows
l_points = np.zeros_like(warped)
r_points = np.zeros_like(warped)
# Go through each level and draw the windows
for level in range(0,len(window_centroids)):
# Window_mask is a function to draw window areas
l_mask = window_mask(window_width,window_height,warped,window_centroids[level][0],level)
r_mask = window_mask(window_width,window_height,warped,window_centroids[level][1],level)
# Add graphic points from window mask here to total pixels found
l_points[(l_points == 255) | ((l_mask == 1) ) ] = 255
r_points[(r_points == 255) | ((r_mask == 1) ) ] = 125#255
# Draw the results
template = np.array(r_points+l_points,np.uint8) # add both left and right window pixels together
zero_channel = np.zeros_like(template) # create a zero color channel
template = np.array(cv2.merge((zero_channel,template,zero_channel)),np.uint8) # make window pixels green
warpage= np.dstack((warped, warped, warped))*255 # making the original road pixels 3 color channels
output = cv2.addWeighted(warpage, 1, template, 0.5, 0.0) # overlay the orignal road image with window results
# If no window centers found, just display orginal road image
else:
output = np.array(cv2.merge((warped,warped,warped)),np.uint8)
return output
# Read in a thresholded image
#warped = mpimg.imread('warped_example.jpg')
test_img = mpimg.imread('./test_images/test2.jpg')
image_threshed = pipeline_thresh(test_img)
binary_warped, M, Minv = pipeline_warp(image_threshed)
# window settings
window_width = 50
window_height = 80 # Break image into 9 vertical layers since image height is 720
margin = 100 # How much to slide left and right for searching
window_centroids = find_window_centroids(binary_warped, window_width, window_height, margin)
output = gen_output(window_centroids, binary_warped)
# Display the final results
plt.imshow(output)
plt.title('window fitting results')
plt.show()
Y_TOP = 450#435#440
Y_BTM = 668
X_CTR = 640#640 = 1280/2
X_TOP_WDT = 120#48#45#50#55#65#50#50#60#60
X_BTM_WDT = 1000#1280#1000#1000#900#850
Y_TOP = 460#435#440
X_TOP_WDT = 150#48#45#50#55#65#50#50#60#60
#X_BTM_WDT = 900
TOP_LEFT = (X_CTR - X_TOP_WDT/2 , Y_TOP)
TOP_RIGHT = (X_CTR + X_TOP_WDT/2, Y_TOP)
BTM_LEFT = (X_CTR - X_BTM_WDT/2 , Y_BTM)
BTM_RIGHT = (X_CTR + X_BTM_WDT/2, Y_BTM)
SRC_TRAPEZOID_ORG_ORDR = [TOP_LEFT, TOP_RIGHT, BTM_RIGHT, BTM_LEFT]
#SRC_TRAPEZOID_WRP = [TOP_LEFT, TOP_RIGHT, BTM_LEFT, BTM_RIGHT]
def show_process_images(image, objpoints, imgpoints, left_fit_prev=None, right_fit_prev=None, sx_thresh=(20, 100)):
#put_text(image, "test")
#cv2.putText(image,text,(start_x,start_y),font, font_size,color)
image = cv2.putText(image,"text",(10,10),fontFace = cv2.FONT_HERSHEY_PLAIN, fontScale = 10, color = (255,255,0))
undistorted = cal_undistort(image, objpoints, imgpoints)
#show_two_images(image, undistorted)
thresed = pipeline_thresh(undistorted, sx_thresh=sx_thresh)
#show_two_images(undistorted, thresed)
binary_warped, M, Minv = pipeline_warp(thresed)
undistorted_w_line = draw_rectangle(undistorted)
#show_two_images(undistorted_w_line, binary_warped)
if not (left_fit_prev==None and right_fit_prev==None):
left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img = find_lines_again(binary_warped, left_fit_prev, right_fit_prev)
else:
left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img = find_lines(binary_warped)
line_projector = LineProjector(Minv)
#visualize_lines(ploty, left_fit, right_fit, left_lane_inds, right_lane_inds, nonzerox, nonzeroy)
result = line_projector.project_lines(undistorted_w_line, binary_warped, left_fit, right_fit)
#result = project_lines(org_image, warped, left_fit, right_fit, ploty, Minv)
#plt.imshow(result)
show_two_images(thresed, binary_warped, title1="Original Image", title2="Warped Image")
empty = None
ploty = np.linspace(0, image.shape[0]-1, image.shape[0] )
y_eval = np.max(ploty)
left_curverad, right_curverad = calc_radius_in_meter(y_eval, left_fit, right_fit)
#print(left_curverad, right_curverad)
vehicle_position = calc_vehicle_position(left_fit, right_fit, binary_warped)
text_radius = "Radius:{} ".format((left_curverad + right_curverad)/2)
text_position = "Vehicle Position:{} ".format(vehicle_position)
#put_text(result, text)
print(text_position)
cv2.putText(result,text_radius,(50,100),fontFace = cv2.FONT_ITALIC, fontScale = 3, color = (0,0,255))#255,255,255))
cv2.putText(result,text_position,(50,200),fontFace = cv2.FONT_ITALIC, fontScale = 3, color = (0,255,0))#255,255,255))
show_two_images(result, result)
left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img = find_lines(binary_warped)
visualize_lines(binary_warped, left_fit, right_fit, left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img)
return left_fit, right_fit
test_img = mpimg.imread('./test_images/straight_lines1.jpg')
#print(test_img.shape[0])
#print(test_img.shape[1])
show_process_images(test_img, objpoints, imgpoints)
test_img = mpimg.imread('./test_images/test1.jpg')
show_process_images(test_img, objpoints, imgpoints)
test_img = mpimg.imread('./test_images/test2.jpg')
show_process_images(test_img, objpoints, imgpoints)
test_img = mpimg.imread('./test_images/test4.jpg')
left_fit_prev, right_fit_prev = show_process_images(test_img, objpoints, imgpoints)
TODO: Consider the following
The approach to better this is to
Try other color spaces. For instance looking at the blue and green channels of the RGB might give you yellow pixels easily (yellow = blue + green right) Try to limit your search based on the expected lane width. This will prevent the weird effects that you see in your final image. In every detection, reinforce the weaker lane (lane with lesser pixels) with the stronger lane by assuming that the two lanes are parallel.
If there are too few pixels detected in a particular lane, just discard it and use the previous lane as the value. One another approach I took was to do a weighted average of the polynomials from previous frames to current one, based on the number of thresholded pixels. So if the current image has a lot of thresholded pixels, then I give the current polynomial a lot of importance, otherwise no. This is a generalization of point 3.
test_img = mpimg.imread('./issue_images/org_video_image_20171217_10_22_06.jpg')
#test_output = pipeline_thresh(test_img, s_thresh=(170, 255), sx_thresh=(5, 100))
show_process_images(test_img, objpoints, imgpoints)#, sx_thresh=(5, 100))
show_process_images(test_img, objpoints, imgpoints,left_fit_prev, right_fit_prev)
flg_in_process = False
left_fit = None
right_fit = None
count = 0
DIR_ORG_VIDEO_SHOT = "./video_images/"
DIR_CVT_VIDEO_SHOT = "./converted_images/"
ORG_VIDEO_FILE_NAME = "org_video_image_"
CVT_VIDEO_FILE_NAME = "cvt_video_image_"
from PIL import Image
def save_image(image, dirname, filename):
"""save a image file"""
filepath = dirname + filename + datetime.now().strftime("%Y%m%d_%H_%M_%S.jpg")
if not os.path.exists(filepath) :
Image.fromarray(image).save(filepath)
from datetime import datetime
import os
def video_pipeline(image):
#save_image(image,DIR_ORG_VIDEO_SHOT, ORG_VIDEO_FILE_NAME)
processed_image = np.copy(image)
# Undistort image (Try)
processed_image = cal_undistort(processed_image, objpoints, imgpoints)
processed_image = pipeline_thresh(processed_image)
binary_warped, M, Minv = pipeline_warp(processed_image)
if flg_in_process == False:
left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img = find_lines(binary_warped)
else:
left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img = find_lines_again(binary_warped, left_fit, right_fit)
#ploty = np.linspace(0, binary_warped.shape[0]-1, binary_warped.shape[0] )
line_projector = LineProjector(Minv)
processed_image = line_projector.project_lines(image, binary_warped, left_fit, right_fit)#, ploty, Minv)
ploty = np.linspace(0, image.shape[0]-1, image.shape[0] )
y_eval = np.max(ploty)
left_curverad, right_curverad = calc_radius_in_meter(y_eval, left_fit, right_fit)
#text = "Radius:{} ".format((left_curverad + right_curverad)/2)
#cv2.putText(processed_image,text,(50,200),fontFace = cv2.FONT_ITALIC, fontScale = 3, color = (0,0,255))#255,255,255))
vehicle_position = calc_vehicle_position(left_fit, right_fit, binary_warped)
text_radius = "Radius:{} ".format((left_curverad + right_curverad)/2)
text_position = "Vehicle Position:{} ".format(vehicle_position)
cv2.putText(processed_image,text_radius,(50,100),fontFace = cv2.FONT_ITALIC, fontScale = 3, color = (0,0,255))#255,255,255))
cv2.putText(processed_image,text_position,(50,200),fontFace = cv2.FONT_ITALIC, fontScale = 3, color = (0,255,0))#255,255,255))
#save_image(processed_image,DIR_CVT_VIDEO_SHOT, CVT_VIDEO_FILE_NAME)
return processed_image # This must be a color image
def process_image(image):
# NOTE: The output you return should be a color image (3 channel) for processing video below
# TODO: put your pipeline here,
# you should return the final output (image where lines are drawn on lanes)
result = video_pipeline(image)
return result
# Import everything needed to edit/save/watch video clips
from moviepy.editor import VideoFileClip
from IPython.display import HTML
VIDEO_INPUT = 'challenge_video.mp4'
VIDEO_OUTPUT = 'output_images/challenge_video_output.mp4'
## To speed up the testing process you may want to try your pipeline on a shorter subclip of the video
## To do so add .subclip(start_second,end_second) to the end of the line below
## Where start_second and end_second are integer values representing the start and end of the subclip
## You may also uncomment the following line for a subclip of the first 5 seconds
##clip1 = VideoFileClip("test_videos/solidWhiteRight.mp4").subclip(0,5)
clip1 = VideoFileClip(VIDEO_INPUT)
white_clip = clip1.fl_image(process_image) #NOTE: this function expects color images!!
%time white_clip.write_videofile(VIDEO_OUTPUT, audio=False)
flg_in_process = False
left_fit = None
right_fit = None
HTML("""
<video width="960" height="540" controls>
<source src="{0}">
</video>
""".format(VIDEO_OUTPUT))